warden_server/README

@@ -42,13 +42,13 @@ B. Dependencies
 
  2. Python modules
 
-    python-mysqldb 5.3.3+
+    python-mysqldb 5.3.3+ | python-psycopg2 2.8.6+
     python-m2crypto 0.20+
     jsonschema 2.4+
 
  3. Database
 
-    MySQL | MariaDB >= 5.5
+    MySQL | MariaDB >= 5.5 | PostgreSQL >= 13
 
 ------------------------------------------------------------------------------
 C. Installation
@@ -70,14 +70,28 @@ C. Installation
    > GRANT ALL ON warden3.* TO `warden`@`localhost`;
    > FLUSH PRIVILEGES;
 
+   # psql
+
+   > CREATE DATABASE warden3 ENCODING 'UTF-8';
+   > CREATE ROLE "warden" LOGIN PASSWORD 'example';
+   > GRANT ALL ON DATABASE "warden3" TO "warden";
+
  * Create necessary table structure
 
-   mysql -p -u warden warden3 < warden_3.0.sql
+   mysql -p -u warden warden3 < warden_3.0_mysql.sql
+
+   or
+
+   psql -U warden -h localhost warden3 < warden_3.0_postgres.sql
 
  * Get up to date Idea schema
 
    wget -O warden_server/idea.schema https://idea.cesnet.cz/_media/en/idea0.schema
 
+ * Load category and tag maps into database (This step is optional for MySQL dbms)
+
+   ./warden_server.py loadmaps
+
  * Enable mod_wsgi, mod_ssl, include Warden configuration
 
    This depends heavily on your distribution and Apache configuration.
@@ -129,7 +143,7 @@ particular implementation object of the aspect, for example type of logger
    Sections and their "type" objects can be:
 
 		Log: FileLogger, SysLogger
-		DB: MySQL
+		DB: MySQL, PostgreSQL
 		Auth: X509Authenticator, X509NameAuthenticator,
               X509MixMatchAuthenticator,PlainAuthenticator
 		Validator: JSONSchemaValidator, NoValidator
@@ -186,22 +200,36 @@ object from particular section list is used ("FileLogger" for example).
       retry_count: number of retries, defaults to 3
       event_size_limit: max size of serialized event, defaults to 5 MB
       catmap_filename: IDEA category mapping to database ids, defaults to
-                       "catmap_mysql.json" at installation directory
+                       "catmap_db.json" at installation directory
+      tagmap_filename": IDEA node type mapping to database ids, defaults to
+                        "tagmap_db.json" at installation directory
+
+   PostgreSQL: database storage backend
+      host: database server host, default "localhost"
+      user: database user, default "warden"
+      password: database password
+      dbname: database name, default "warden3"
+      port: database server port, default 5432
+      retry_pause: retry in case of database errors, in seconds, defaults to 5
+      retry_count: number of retries, defaults to 3
+      event_size_limit: max size of serialized event, defaults to 5 MB
+      catmap_filename: IDEA category mapping to database ids, defaults to
+                       "catmap_db.json" at installation directory
       tagmap_filename": IDEA node type mapping to database ids, defaults to
-                        "tagmap_mysql.json" at installation directory
+                        "tagmap_db.json" at installation directory
 
    WardenHandler: Main Warden RPC worker
       send_events_limit: max events sent in one bunch, defaults to 10000
       get_events_limit: max events received in one bunch, defaults to 10000
       description: human readable description, sent in server info
-	
+
 ------------------------------------------------------------------------------
 E. Command line
 
    When run from command line, server offers set of commands and options for
 runtime and database management. You can also use --help option for each
 command and for server itself.
-   
+
 warden_server.py [--help] [-c CONFIG] <command>
 
 optional arguments:
@@ -285,11 +313,9 @@ warden_server.py purge [--help] [-l] [-e] [-d DAYS]
 
 warden_server.py loadmaps [--help]
 
-   Load 'categories' and 'tags' table from 'catmap_mysql.json' and
-   'tagmap_mysql.json'. Note that this is NOT needed for server at all, load
-   them into db at will, should you need to run your own specific SQL queries
-   on data directly. Note also that previous content of both tables will be
-   lost.
+   Load 'categories' and 'tags' table from 'catmap_db.json' and
+   'tagmap_db.json'. Note also that previous content of both tables
+   will be lost.
 
 ------------------------------------------------------------------------------
 Copyright (C) 2011-2015 Cesnet z.s.p.o

warden_server/test_warden_server.py

@@ -39,7 +39,7 @@ def setUpModule():      # pylint: disable = locally-disabled, invalid-name
             cur.execute("DROP DATABASE IF EXISTS %s" % (DB,))   # NOT SECURE
             cur.execute("CREATE DATABASE %s" % (DB,))           # NOT SECURE
             cur.execute("USE %s" % (DB,))                       # NOT SECURE
-            with open(path.join(path.dirname(__file__), 'warden_3.0.sql')) as script:
+            with open(path.join(path.dirname(__file__), 'warden_3.0_mysql.sql')) as script:
                 statements = ''.join([line.replace('\n', '') for line in script if line[0:2] != '--']).split(';')[:-1]
                 for statement in statements:
                     cur.execute(statement)

warden_server/warden_3.0_postgres.sql

+SET TimeZone='+00:00';
+
+CREATE COLLATION case_insensitive (
+  provider = icu,
+  locale = 'und-u-ks-level2',
+  deterministic = false
+);
+
+-- ---------------------------------------------------------
+
+--
+-- Database: "warden3"
+--
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table "categories"
+--
+
+CREATE TABLE "categories" (
+  "id" int NOT NULL UNIQUE CHECK ("id" >= 0),
+  "category" text NOT NULL COLLATE case_insensitive,
+  "subcategory" text DEFAULT NULL COLLATE case_insensitive,
+  "cat_subcat" text NOT NULL COLLATE case_insensitive
+);
+
+CREATE INDEX "cat_sub" ON "categories" ("cat_subcat");
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table "clients"
+--
+
+CREATE TABLE "clients" (
+  "id" SERIAL PRIMARY KEY,
+  "registered" timestamp NOT NULL DEFAULT '1970-01-01 00:00:00',
+  "requestor" text NOT NULL COLLATE case_insensitive,
+  "hostname" text NOT NULL COLLATE case_insensitive,
+  "note" text NULL COLLATE case_insensitive,
+  "valid" smallint NOT NULL DEFAULT '1' CHECK ("valid" >= 0),
+  "name" text NOT NULL,
+  "secret" text NULL,
+  "read" smallint NOT NULL DEFAULT '1' CHECK ("read" >= 0),
+  "debug" smallint NOT NULL DEFAULT '0' CHECK ("debug" >= 0),
+  "write" smallint NOT NULL DEFAULT '0' CHECK ("write" >= 0),
+  "test" smallint NOT NULL DEFAULT '0' CHECK ("test" >= 0)
+);
+
+CREATE INDEX "clients_1" ON "clients" ("valid", "secret", "hostname");
+CREATE INDEX "clients_2" ON "clients" ("valid", "name");
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table "events"
+--
+
+CREATE TABLE "events" (
+  "id" BIGSERIAL PRIMARY KEY,
+  "received" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  "client_id" int NOT NULL REFERENCES "clients" ("id"),
+  "data" bytea NOT NULL,
+  "valid" smallint NOT NULL DEFAULT '1' CHECK ("valid" >= 0)
+);
+
+CREATE INDEX "id" ON "events" ("id", "client_id");
+CREATE INDEX "received" ON "events" ("received");
+
+SELECT nextval('events_id_seq'); -- AUTO_INCREMENT = 2
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table "event_category_mapping"
+--
+
+CREATE TABLE "event_category_mapping" (
+  "event_id" bigint NOT NULL REFERENCES "events" ("id") ON DELETE CASCADE,
+  "category_id" int NOT NULL,
+  PRIMARY KEY ("event_id", "category_id"),
+  CONSTRAINT "event_category_mapping_category_id_fk" FOREIGN KEY ("category_id") REFERENCES "categories" ("id")
+);
+
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table "last_events"
+--
+
+CREATE TABLE "last_events" (
+  "id" SERIAL PRIMARY KEY,
+  "client_id" int NOT NULL REFERENCES "clients" ("id"),
+  "event_id" bigint REFERENCES "events" ("id"),
+  "timestamp" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX "client_id" ON "last_events" ("client_id", "event_id");
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table "tags"
+--
+
+CREATE TABLE "tags" (
+  "id" int NOT NULL UNIQUE CHECK ("id" >= 0),
+  "tag" text NOT NULL COLLATE case_insensitive
+);
+
+CREATE INDEX "id_tag_name" ON "tags" ("id", "tag");
+CREATE INDEX "tag_name" ON "tags" ("tag");
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table "event_tag_mapping"
+--
+
+CREATE TABLE "event_tag_mapping" (
+  "event_id" bigint NOT NULL REFERENCES "events" ("id") ON DELETE CASCADE,
+  "tag_id" int NOT NULL,
+  PRIMARY KEY ("event_id", "tag_id"),
+  CONSTRAINT "event_tag_mapping_tag_id_fk" FOREIGN KEY ("tag_id") REFERENCES "tags" ("id")
+);