warden3/contrib/connectors/hp-kippo/warden3-kippo-sender.py → contrib/connectors/hp-kippo/warden3-kippo-sender.py

@@ -93,13 +93,13 @@ def main():
   crs = con.cursor()
 
   events = []
-  query =  "SELECT UNIX_TIMESTAMP(CONVERT_TZ(s.starttime, '+00:00', @@global.time_zone)) as starttime, s.ip, COUNT(s.id) as attack_scale, sn.ip as sensor \
+  query =  "SELECT MIN(UNIX_TIMESTAMP(s.starttime)) as starttime, s.ip, COUNT(s.id) as attack_scale, sn.ip as sensor \
             FROM sessions s \
             LEFT JOIN sensors sn ON s.sensor=sn.id \
-            WHERE s.starttime > DATE_SUB(UTC_TIMESTAMP(), INTERVAL + %s SECOND) \
-            GROUP BY s.ip ORDER BY s.starttime ASC;"
+            WHERE s.starttime > DATE_SUB(CURRENT_TIMESTAMP(), INTERVAL + %s SECOND) \
+            GROUP BY s.ip, sn.ip ORDER BY starttime ASC;"
 
-  crs.execute(query, awin)
+  crs.execute(query, (awin,))
   rows = crs.fetchall()
   for row in rows:
     dtime = format_timestamp(row['starttime'])

warden3/contrib/connectors/hp-labrea/labrea-idea → contrib/connectors/hp-labrea/labrea-idea

@@ -31,7 +31,7 @@ ACTION="$1"
 
 case "$ACTION" in
 	start)
-		if [ -z "PARAMS" ]; then
+		if [ -z "$PARAMS" ]; then
 			log_daemon_msg "Unconfigured $DAEMON_NAME, not starting."
 			exit 2
 		fi

warden3/contrib/connectors/hp-labrea/labrea-idea.py → contrib/connectors/hp-labrea/labrea-idea.py

@@ -34,6 +34,9 @@ class WindowContextMgr(object):
         self.ideagen = ideagen
         self.first_update_queue = OrderedDict()
         self.last_update_queue = OrderedDict()
+        # Hammer to mitigate too big events
+        self.max_count = 2000
+        self.max_src_ports = 1024
 
     def expire_queue(self, queue, window):
         aggr_events = []
@@ -68,9 +71,16 @@ class WindowContextMgr(object):
                     self.first_update_queue[ctx] = self.update_timestamp
                     self.last_update_queue[ctx] = self.update_timestamp
                 else:
-                    self.ctx_append(self.contexts[ctx], event)
-                    del self.last_update_queue[ctx]
-                    self.last_update_queue[ctx] = self.update_timestamp
+                    if not self.ctx_append(self.contexts[ctx], event):
+                        closed = self.ctx_close(self.contexts[ctx])
+                        if closed is not None:
+                            aggr_events.append(closed)
+                        del self.contexts[ctx]
+                        del self.first_update_queue[ctx]
+                        del self.last_update_queue[ctx]
+                    else:
+                        del self.last_update_queue[ctx]
+                        self.last_update_queue[ctx] = self.update_timestamp
 
         return aggr_events
 
@@ -107,6 +117,7 @@ class PingContextMgr(WindowContextMgr):
         ctx["tgt_ips"].add(event.tgt_ip)
         ctx["count"] += 1
         ctx["last_update"] = self.update_timestamp
+        return ctx["count"] < self.max_count
 
     def ctx_close(self, ctx):
         return self.ideagen.gen_idea(
@@ -143,11 +154,13 @@ class ConnectContextMgr(WindowContextMgr):
         ctx["src_ports"].add(event.src_port)
         ctx["count"] += 1
         ctx["last_update"] = self.update_timestamp
+        return ctx["count"] < self.max_count
 
     def ctx_close(self, ctx):
+        src_ports = ctx["src_ports"] if len(ctx["src_ports"]) <= self.max_src_ports else None
         return self.ideagen.gen_idea(
             src=ctx["src_ip"],
-            src_ports=ctx["src_ports"],
+            src_ports=src_ports,
             targets=ctx["tgt_ips_ports"].items(),
             detect_time=self.update_timestamp,
             event_time=ctx["first_update"],
@@ -419,10 +432,6 @@ def daemonize(
                 os.close(fd)
             except Exception:
                 pass
-    # Redirect stdin, stdout, stderr to /dev/null
-    devnull = os.open(os.devnull, os.O_RDWR)
-    for fd in range(3):
-        os.dup2(devnull, fd)
     # PID file
     if pidfile is not None:
         pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
@@ -436,6 +445,10 @@ def daemonize(
                 os.unlink(pidfile)
             except Exception:
                 pass
+    # Redirect stdin, stdout, stderr to /dev/null
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for fd in range(3):
+        os.dup2(devnull, fd)
 
 
 def save_events(aggr, filer):

contrib/map/LICENSE

+BSD License
+
+Copyright © 2016 Cesnet z.s.p.o
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the Cesnet z.s.p.o nor the names of its
+      contributors may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE Cesnet z.s.p.o BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

contrib/map/README

++---------------------------------+
+| Warden Map Client 1.0           |
++---------------------------------+
+
+Content
+
+  A. Introduction
+  B. Configuration
+  C. Usage & Help
+
+------------------------------------------------------------------------------
+A. Introduction
+
+  Warden Map Client is very simple client for drawing a map with events from
+  database of the Warden server. It consists of a Python 2.7 backend and
+  a javascript/jquery frontend.
+
+  Backend uses Warden API for downloading of events. Events are processed and
+  enhanced with a geodata via freegeoip.net API. Finally warden-map.json file
+  with information for the frontend is created.
+
+  Frontend uses datamaps project (http://datamaps.github.io/) for visualisation
+  of events on a map. It is possible to check details of the event by moving
+  cursor on a arc. It is also possible to zoom map via scrolling and/or clicking
+  on the plus, minus and, home buttons.
+
+------------------------------------------------------------------------------
+B. Configuration
+
+  1. Copy frontend folder into desired location.
+
+  2. Copy html snippet into your web page, or use it as an iframe.
+     NOTE: If necessary, change css/js paths in a html snippet.
+
+  3. Copy backend folder into desired location.
+
+  4. Setup backend call (warden-map.py) in a crontab.
+     NOTE: Please make sure you will have stored warden-map.json file
+           in the frontend folder.
+     EXAMPLE: ./warden-map.py --client cz.cesnet.warden.map \
+                              --key certs/key.pem \
+                              --cert certs/cert.pem \
+                              --output ../frontend/
+
+  5. Enjoy your map.
+
+------------------------------------------------------------------------------
+C. Usage & Help
+
+usage: warden-map.py [-h] [--output /path/] --events <number> --client
+                     <org.ex.cl> --key /path/key.pem --cert /path/cert.pem
+                     --cacert /path/cacert.pem --secret <SeCreT>
+
+optional arguments:
+  -h, --help                 show this help message and exit
+  --output path/            path where warden-map.json should be saved
+
+required arguments:
+  --events <number>          count of events for a map
+  --client <org.ex.cl>       client name
+  --key path/key.pem        SSL key for a client
+  --cert path/cert.pem      SSL cert for a client
+  --cacert path/cacert.pem  SSL cacert for a client
+  --secret <SeCreT>          secret key for a client
+
+
+------------------------------------------------------------------------------
+Copyright (C) 2016 Cesnet z.s.p.o

contrib/map/backend/warden-map.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# warden-map.py
+#
+# Copyright (C) 2016 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+import json
+import codecs
+import time
+import argparse
+import GeoIP
+import requests
+
+def getLastEvents(client, key, cert):
+
+  res = requests.post(
+      'https://warden-hub.cesnet.cz/warden3/getEvents?client=%s' % (client,),
+      cert=(cert, key)
+  )
+
+  data = res.json()
+  i = 0
+  eventsList = []
+  for p in data['events']:
+    event = {}
+    for key, value in { 'event': 'Category', 'time': 'DetectTime', 'origin': 'Source', 'destination': 'Target'}.items():
+      if value in p:
+        if (key == 'origin') or (key == 'destination'):
+          event[key] = {}
+          if 'IP4' in p[value][0]:
+            event[key]['ip'] = p[value][0]['IP4'][0]
+          else:
+            event[key] = {}
+        elif (key == 'event'):
+          event[key] = ', '.join(p[value])
+        else:
+          event[key] = p[value]
+      else:
+        if (key == 'origin') or (key == 'destination'):
+          event[key] = {}
+        else:
+          event[key] = {}
+    if 'ip' in event['origin']:
+      eventsList.append(event)
+      i += 1
+
+  return eventsList
+
+def getGeolocation(ip, db):
+
+  data = db.record_by_addr(ip)
+
+  if not data:
+    return {}
+  else:
+    return {
+      'latitude': data['latitude'], 
+      'longitude': data['longitude'], 
+      'country_name': data['country_name'] if data['country_name'] else None, 
+      'city': data['city'] if data['city'] else None
+    }
+
+def main(args):
+
+  client = args.client[0]
+  key    = args.key[0]
+  cert   = args.cert[0]
+
+  if args.output is not None:
+    path = args.output[0] + 'warden-map.json'
+  else:
+    path = 'warden-map.json'
+
+  db = GeoIP.open("GeoLiteCity.dat", GeoIP.GEOIP_MEMORY_CACHE)
+  db.set_charset(GeoIP.GEOIP_CHARSET_UTF8)
+
+  wardenEvents = getLastEvents(client, key, cert)
+
+  for p in wardenEvents:
+    for target in {'origin', 'destination'}:
+      geoData = {}
+      if 'ip' in p[target]:
+        geoData = getGeolocation(p[target]['ip'], db)
+        for value in {'latitude', 'longitude', 'country_name', 'city'}:
+          if value in geoData:
+            if not geoData[value]:
+              p[target][value] = "???"
+            else:
+              p[target][value] = geoData[value]
+          else:
+            p[target][value] = "???"
+
+      else:
+        p[target]['ip'] = "???"
+        p[target]['country_name'] = "Czech Republic"
+        p[target]['city'] = "???"
+        p[target]['latitude'] = 49.743
+        p[target]['longitude'] = 15.338
+
+  wardenEvents.append(int(time.time()));
+
+  with open(path, 'w') as outfile:
+    json.dump(wardenEvents, outfile)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='Creates warden-map.json for warden-map.html frontend.',
+                                  formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=30))
+
+  parser.add_argument('--output', metavar='path/', type=str,
+                    nargs=1, help='path where warden-map.json should be saved')
+
+  requiredNamed = parser.add_argument_group('required arguments')
+
+  requiredNamed.add_argument('--client', metavar='<org.ex.cl>', type=str, required=True,
+                    nargs=1, help='client name')
+  requiredNamed.add_argument('--key', metavar='path/key.pem', type=str, required=True,
+                    nargs=1, help='SSL key for a client')
+  requiredNamed.add_argument('--cert', metavar='path/cert.pem', type=str, required=True,
+                    nargs=1, help='SSL cert for a client')
+
+  args = parser.parse_args()
+  main(args)