diff --git a/warden3/warden_server/warden3.0-alpha.sql b/warden3/warden_server/warden3.0-alpha.sql new file mode 100644 index 0000000000000000000000000000000000000000..7fa656710ac604a47795ece7fcbf3633dabca29e --- /dev/null +++ b/warden3/warden_server/warden3.0-alpha.sql @@ -0,0 +1,277 @@ +-- phpMyAdmin SQL Dump +-- version 3.4.11.1deb2+deb7u1 +-- http://www.phpmyadmin.net +-- +-- Host: localhost +-- Generation Time: Dec 04, 2014 at 02:54 PM +-- Server version: 5.5.38 +-- PHP Version: 5.4.4-14+deb7u14 + +SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO"; +SET time_zone = "+00:00"; + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; + +-- +-- Database: `warden3` +-- + +-- -------------------------------------------------------- + +-- +-- Table structure for table `categories` +-- + +CREATE TABLE IF NOT EXISTS `categories` ( + `id` int(11) NOT NULL, + `category` varchar(64) NOT NULL, + `subcategory` varchar(64) DEFAULT NULL, + `cat_subcat` varchar(129) NOT NULL, + KEY `cat_sub` (`cat_subcat`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; + +-- +-- Dumping data for table `categories` +-- + +INSERT INTO `categories` (`id`, `category`, `subcategory`, `cat_subcat`) VALUES +(101, 'Abusive', 'Spam', 'Abusive.Spam'), +(102, 'Abusive', 'Harassment', 'Abusive.Harassment'), +(103, 'Abusive', 'Child', 'Abusive.Child'), +(104, 'Abusive', 'Sexual', 'Abusive.Sexual'), +(105, 'Abusive', 'Violence', 'Abusive.Violence'), +(201, 'Malware', 'Virus', 'Malware.Virus'), +(202, 'Malware', 'Worm', 'Malware.Worm'), +(203, 'Malware', 'Trojan', 'Malware.Trojan'), +(204, 'Malware', 'Spyware', 'Malware.Spyware'), +(205, 'Malware', 'Dialer', 'Malware.Dialer'), +(206, 'Malware', 'Rootkit', 'Malware.Rootkit'), +(301, 'Recon', 'Scanning', 'Recon.Scanning'), +(302, 'Recon', 'Sniffing', 'Recon.Sniffing'), +(303, 'Recon', 'SocialEngineering', 'Recon.SocialEngineering'), +(304, 'Recon', 'Searching', 'Recon.Searching'), +(401, 'Attempt', 'Exploit', 'Attempt.Exploit'), +(402, 'Attempt', 'Login', 'Attempt.Login'), +(403, 'Attempt', 'NewSignature', 'Attempt.NewSignature'), +(501, 'Intrusion', 'AdminCompromise', 'Intrusion.AdminCompromise'), +(502, 'Intrusion', 'UserCompromise', 'Intrusion.UserCompromise'), +(503, 'Intrusion', 'AppCompromise', 'Intrusion.AppCompromise'), +(504, 'Intrusion', 'Botnet', 'Intrusion.Botnet'), +(601, 'Availability', 'DoS', 'Availability.DoS'), +(602, 'Availability', 'DDoS', 'Availability.DDoS'), +(603, 'Availability', 'Sabotage', 'Availability.Sabotage'), +(604, 'Availability', 'Outage', 'Availability.Outage'), +(701, 'Information', 'UnauthorizedAccess', 'Information.UnauthorizedAccess'), +(702, 'Information', 'UnauthorizedModification', 'Information.UnauthorizedModification'), +(801, 'Fraud', 'UnauthorizedUsage', 'Fraud.UnauthorizedUsage'), +(802, 'Fraud', 'Copyright', 'Fraud.Copyright'), +(803, 'Fraud', 'Masquerade', 'Fraud.Masquerade'), +(804, 'Fraud', 'Phishing', 'Fraud.Phishing'), +(805, 'Fraud', 'Scam', 'Fraud.Scam'), +(901, 'Vulnerable', 'Open', 'Vulnerable.Open'), +(1001, 'Anomaly', 'Traffic', 'Anomaly.Traffic'), +(1002, 'Anomaly', 'Connection', 'Anomaly.Connection'), +(1003, 'Anomaly', 'Protocol', 'Anomaly.Protocol'), +(1004, 'Anomaly', 'System', 'Anomaly.System'), +(1005, 'Anomaly', 'Application', 'Anomaly.Application'), +(1006, 'Anomaly', 'Behaviour', 'Anomaly.Behaviour'), +(9998, 'Other', '', 'Other'), +(9999, 'Test', '', 'Test'); + +-- -------------------------------------------------------- + +-- +-- Table structure for table `clients` +-- + +CREATE TABLE IF NOT EXISTS `clients` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `hostname` varchar(256) NOT NULL, + `registered` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `requestor` varchar(256) NOT NULL, + `note` text NOT NULL, + `valid` tinyint(1) NOT NULL DEFAULT '1', + `read` tinyint(1) NOT NULL DEFAULT '0', + `debug` tinyint(1) NOT NULL DEFAULT '0', + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=29 ; + +-- +-- Dumping data for table `clients` +-- + +INSERT INTO `clients` (`id`, `hostname`, `registered`, `requestor`, `note`, `valid`, `read`, `debug`) VALUES +(1, 'afrodita.civ.zcu.cz', '0000-00-00 00:00:00', '', '', 1, 1, 1), +(3, 'au1.cesnet.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(12, 'au2.cesnet.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(13, 'bee.net.vutbr.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(14, 'buldog.vsb.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(15, 'collector-nemea.liberouter.org', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(17, 'collector.liberouter.org', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(18, 'holly.cesnet.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(19, 'kryten.cesnet.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(20, 'mentat.cesnet.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(21, 'miel.opf.slu.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(23, 'nfsen.ics.muni.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(25, 'vinovago.cesnet.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(26, 'ward.tul.cz', '0000-00-00 00:00:00', '', '', 1, 1, 0), +(28, 'kostik.zcu.cz', '0000-00-00 00:00:00', 'kostenec@civ.zcu.cz', '', 1, 1, 1); + +-- -------------------------------------------------------- + +-- +-- Table structure for table `events` +-- + +CREATE TABLE IF NOT EXISTS `events` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `detected_epoch` int(11) NOT NULL, + `received` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `detected` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `client_id` int(11) NOT NULL, + `data` text NOT NULL, + `valid` tinyint(1) NOT NULL DEFAULT '1', + `origin` varchar(64) NOT NULL, + `origin_hash` varchar(32) NOT NULL, + `origin_long` varchar(64) NOT NULL, + PRIMARY KEY (`id`), + KEY `detected` (`detected`), + KEY `id` (`id`,`client_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `event_category_mapping` +-- + +CREATE TABLE IF NOT EXISTS `event_category_mapping` ( + `event_id` int(11) NOT NULL, + `category_id` int(11) NOT NULL, + KEY `event_id_2` (`event_id`,`category_id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `event_tag_mapping` +-- + +CREATE TABLE IF NOT EXISTS `event_tag_mapping` ( + `event_id` int(11) NOT NULL, + `tag_id` int(11) NOT NULL, + KEY `event_id_2` (`event_id`,`tag_id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `last_events` +-- + +CREATE TABLE IF NOT EXISTS `last_events` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `client_id` int(11) NOT NULL, + `event_id` int(11) NOT NULL, + `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `request` text NOT NULL, + PRIMARY KEY (`id`), + KEY `client_id` (`client_id`,`event_id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `services` +-- + +CREATE TABLE IF NOT EXISTS `services` ( + `service_id` int(11) NOT NULL AUTO_INCREMENT, + `client_id` int(11) NOT NULL, + `registered` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `requestor` varchar(256) NOT NULL, + `service` varchar(256) NOT NULL, + `note` text NOT NULL, + `valid` tinyint(1) NOT NULL DEFAULT '1', + `identity` varchar(64) NOT NULL, + `write` tinyint(1) NOT NULL DEFAULT '0', + `test` int(11) NOT NULL DEFAULT '0', + PRIMARY KEY (`service_id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=31 ; + +-- +-- Dumping data for table `services` +-- + +INSERT INTO `services` (`service_id`, `client_id`, `registered`, `requestor`, `service`, `note`, `valid`, `identity`, `write`, `test`) VALUES +(1, 1, '0000-00-00 00:00:00', '', 'hihat', '', 1, 'cz.zcu.civ.afrodita.hihat', 1, 0), +(2, 1, '0000-00-00 00:00:00', '', 'labrea', '', 1, 'cz.zcu.civ.afrodita.labrea', 1, 0), +(3, 3, '0000-00-00 00:00:00', '', 'cesnet_ids', '', 1, 'cz.cesnet.au1.cesnet_ids', 1, 0), +(4, 3, '0000-00-00 00:00:00', '', 'cesnet_sserv', '', 1, 'cz.cesnet.au1.cesnet_sserv', 1, 0), +(5, 3, '0000-00-00 00:00:00', '', 'n6d-dorkbot', '', 1, 'cz.cesnet.au1.n6d-dorkbot', 1, 0), +(6, 3, '0000-00-00 00:00:00', '', 'n6e-certplsinkhole', '', 1, 'cz.cesnet.au1.n6e-certplsinkhole', 1, 0), +(7, 3, '0000-00-00 00:00:00', '', 'n6i-citadelsinkhole', '', 1, 'cz.cesnet.au1.n6i-citadelsinkhole', 1, 0), +(8, 3, '0000-00-00 00:00:00', '', 'n6i-citadelsinkholeqd', '', 1, 'cz.cesnet.au1.n6i-citadelsinkholeqd', 1, 0), +(9, 3, '0000-00-00 00:00:00', '', 'n6n-openntp', '', 1, 'cz.cesnet.au1.n6n-openntp', 1, 0), +(10, 3, '0000-00-00 00:00:00', '', 'n6o-botszeroaccess', '', 1, 'cz.cesnet.au1.n6o-botszeroaccess', 1, 0), +(11, 3, '0000-00-00 00:00:00', '', 'report_n6v-virut', '', 1, 'cz.cesnet.au1.report_n6v-virut', 1, 0), +(12, 12, '0000-00-00 00:00:00', '', 'ids-cz', '', 1, 'cz.cesnet.au2.ids-cz', 1, 0), +(13, 13, '0000-00-00 00:00:00', '', 'hpscan', '', 1, 'cz.vutbr.net.bee.hpscan', 1, 0), +(14, 14, '0000-00-00 00:00:00', '', 'kippo', '', 1, 'cz.vsb.buldog.kippo', 1, 0), +(15, 15, '0000-00-00 00:00:00', '', 'nemea', '', 1, 'org.liberouter.collector-nemea.nemea', 1, 0), +(16, 16, '0000-00-00 00:00:00', '', 'hoststats', '', 1, 'org.liberouter.collector.hoststats', 1, 0), +(17, 17, '0000-00-00 00:00:00', '', 'synscandetector_1_0', '', 1, 'org.liberouter.collector.synscandetector_1_0', 1, 0), +(18, 18, '0000-00-00 00:00:00', '', 'kippohoneypot', '', 1, 'cz.cesnet.holly.kippohoneypot', 1, 0), +(19, 19, '0000-00-00 00:00:00', '', 'dionaeahoneypot', '', 1, 'cz.cesnet.kryten.dionaeahoneypot', 1, 0), +(20, 29, '0000-00-00 00:00:00', '', 'mentat', '', 1, 'cz.cesnet.mentat.mentat', 1, 0), +(21, 21, '0000-00-00 00:00:00', '', 'kippo', '', 1, 'cz.slu.opf.miel.kippo', 1, 0), +(22, 23, '0000-00-00 00:00:00', '', 'honeyscan', '', 1, 'cz.muni.ics.nfsen.honeyscan', 1, 0), +(23, 23, '0000-00-00 00:00:00', '', 'scandetector_1_0', '', 1, 'cz.muni.ics.nfsen.scandetector_1_0', 1, 0), +(24, 23, '0000-00-00 00:00:00', '', 'sshbruteforce-1_n', '', 1, 'cz.muni.ics.nfsen.sshbruteforce-1_n', 1, 0), +(25, 25, '0000-00-00 00:00:00', '', 'fail2ban', '', 1, 'cz.cesnet.vinovago.fail2ban', 1, 0), +(26, 26, '0000-00-00 00:00:00', '', 'dionaeatul', '', 1, 'cz.tul.ward.dionaeatul', 1, 0), +(27, 26, '0000-00-00 00:00:00', '', 'kippo', '', 1, 'cz.tul.ward.kippo', 1, 0), +(28, 28, '0000-00-00 00:00:00', 'kostenec@civ.zcu.cz', 'com.example.test-node', '', 1, 'com.example.test-node', 0, 0), +(29, 28, '0000-00-00 00:00:00', 'kostenec@civ.zcu.cz', 'com.example.test-node2', '', 1, 'com.example.test-node2', 0, 0), +(30, 28, '0000-00-00 00:00:00', 'kostenec@civ.zcu.cz', 'Test', '', 1, 'com.example.test-node3', 0, 0); + +-- -------------------------------------------------------- + +-- +-- Table structure for table `tags` +-- + +CREATE TABLE IF NOT EXISTS `tags` ( + `id` int(11) NOT NULL, + `tag` varchar(64) NOT NULL, + KEY `id_tag_name` (`id`,`tag`), + KEY `tag_name` (`tag`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; + +-- +-- Dumping data for table `tags` +-- + +INSERT INTO `tags` (`id`, `tag`) VALUES +(1, 'Connection'), +(2, 'Datagram'), +(3, 'Content'), +(4, 'Data'), +(5, 'File'), +(6, 'Flow'), +(7, 'Log'), +(8, 'Protocol'), +(9, 'Host'), +(10, 'Network'), +(11, 'Correlation'), +(12, 'External'), +(13, 'Reporting'), +(99, 'Other'); + +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; diff --git a/warden3/warden_server/warden_server.py b/warden3/warden_server/warden_server.py index 11ce067f26d26fa900d86fd80f76adafe0556adf..e05fdbae92d72c7bb19135dec62c9eb0ad852d8b 100755 --- a/warden3/warden_server/warden_server.py +++ b/warden3/warden_server/warden_server.py @@ -275,7 +275,7 @@ class MySQL(Object): def get_client_by_name(self, name): format_strings = ','.join(['%s'] * len(name)) - self.crs.execute("SELECT cl.`id`, cl.`hostname`, s.`service`, s.`service_id`, s.`identity`, cl.`read`, s.`write`, s.`test`, cl.`debug` FROM `clients3` cl LEFT JOIN `services` s ON cl.`id` = s.`client_id` WHERE cl.`valid` = 1 AND s.`valid` = 1 AND `hostname` IN (%s)" % format_strings, tuple(name)) + self.crs.execute("SELECT cl.`id`, cl.`hostname`, s.`service`, s.`service_id`, s.`identity`, cl.`read`, s.`write`, s.`test`, cl.`debug` FROM `clients` cl LEFT JOIN `services` s ON cl.`id` = s.`client_id` WHERE cl.`valid` = 1 AND s.`valid` = 1 AND `hostname` IN (%s)" % format_strings, tuple(name)) row = self.crs.fetchall() return row if row else None @@ -335,7 +335,7 @@ class MySQL(Object): for pcats in parent_cats: sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats) - sqlwhere.append("e.id IN (SELECT event_id FROM event_category_mapping4 WHERE %s)" % sqltemp['cat']) + sqlwhere.append("e.id IN (SELECT event_id FROM event_category_mapping WHERE %s)" % sqltemp['cat']) if tag is not None and notag is not None: raise Error("Unrealizable conditions. Choose tag or notag option.", 500, method='getEvents', @@ -347,7 +347,7 @@ class MySQL(Object): if notag is not None: sqltemp['tag'] = generateDynamicQuery(self, "Tag", "tag_id NOT IN (%s)", json.loads(notag)) - sqlwhere.append("e.id IN (SELECT event_id FROM event_tag_mapping4 WHERE %s)" % sqltemp['tag']) + sqlwhere.append("e.id IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % sqltemp['tag']) if group is not None and nogroup is not None: @@ -374,8 +374,8 @@ class MySQL(Object): #sqlwhere = sqlwhere[:-4] and_op = "" if not sqlwhere_string else "AND" - logging.debug("SELECT e.id, e.data FROM clients2 cl RIGHT JOIN events4 e ON cl.id = e.client_id WHERE e.id > %s AND %s %s e.valid = 1 LIMIT %s" % (str(id), sqlwhere_string, and_op, str(count))) - self.crs.execute("SELECT e.id, e.data FROM clients2 cl RIGHT JOIN events4 e ON cl.id = e.client_id WHERE e.id > %s AND %s %s e.valid = 1 LIMIT %s" % (str(id), sqlwhere_string, and_op, str(count))) + # logging.debug("SELECT e.id, e.data FROM clients cl RIGHT JOIN events e ON cl.id = e.client_id WHERE e.id > %s AND %s %s e.valid = 1 LIMIT %s" % (str(id), sqlwhere_string, and_op, str(count))) + self.crs.execute("SELECT e.id, e.data FROM clients cl RIGHT JOIN events e ON cl.id = e.client_id WHERE e.id > %s AND %s %s e.valid = 1 LIMIT %s" % (str(id), sqlwhere_string, and_op, str(count))) row = self.crs.fetchall() if row: @@ -397,22 +397,22 @@ class MySQL(Object): for event in events: try: - # logging.debug("INSERT INTO events5 (detected,received,client_id,data) VALUES ('%s', NOW(), '%s', '%s')" % (event['DetectTime'], client['id'], self.con.escape_string(str(event)))) - self.crs.execute("INSERT INTO events5 (detected,received,client_id,data) VALUES ('%s', NOW(), '%s', '%s')" % (event['DetectTime'], client['id'], self.con.escape_string(str(event)))) + # logging.debug("INSERT INTO events (detected,received,client_id,data) VALUES ('%s', NOW(), '%s', '%s')" % (event['DetectTime'], client['id'], self.con.escape_string(str(event)))) + self.crs.execute("INSERT INTO events (detected,received,client_id,data) VALUES ('%s', NOW(), '%s', '%s')" % (event['DetectTime'], client['id'], self.con.escape_string(str(event)))) lastid = self.crs.lastrowid # logging.debug(str(lastid)) for cat in event['Category']: # logging.debug({'cat': cat}) cat_id = self.map_id('Category', cat) if self.map_id('Category', cat) else self.map_id('Category', 'Other.Other') # logging.debug({'cat_id': cat_id}) - # logging.debug("INSERT INTO event_category_mapping5 (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id))) - self.crs.execute("INSERT INTO event_category_mapping5 (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id))) + # logging.debug("INSERT INTO event_category_mapping (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id))) + self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id))) for tag in event['Node'][0]['Tags']: tag_id = self.map_id('Tag', tag) if self.map_id('Tag', tag) else self.map_id('Tag', 'Other') # logging.debug({'tag_id': tag_id}) - # logging.debug("INSERT INTO event_tag_mapping5 (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), tag_id)) - self.crs.execute("INSERT INTO event_tag_mapping5 (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), str(tag_id))) + # logging.debug("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), tag_id)) + self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), str(tag_id))) self.con.commit() except: @@ -427,7 +427,7 @@ class MySQL(Object): self.crs.execute("INSERT INTO last_events(client_id, event_id, timestamp) VALUES(%s, %s, NOW())" % (str(client[0]['id']), id)) def getLastEventId(self): - self.crs.execute("SELECT MAX(id) as id FROM events4") + self.crs.execute("SELECT MAX(id) as id FROM events") row = self.crs.fetchone() return row['id'] if row['id'] is not None else 0 @@ -903,7 +903,7 @@ def build_server(conf): "host": {"type": str, "default": "localhost"}, "user": {"type": str, "default": "warden"}, "password": {"type": str, "default": ""}, - "dbname": {"type": str, "default": "warden3c"}, + "dbname": {"type": str, "default": "warden3"}, "port": {"type": natural, "default": 3306} }, "WardenHandler": {