README added

Optional malware reporting
Native warden3 time functions used
Minor updates

warden3/contrib/connectors/hp-dio/LICENSE

+BSD License
+
+Copyright © 2011-2015 Cesnet z.s.p.o
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the Cesnet z.s.p.o nor the names of its
+      contributors may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE Cesnet z.s.p.o BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

warden3/contrib/connectors/hp-dio/README

++---------------------------------------------+
+| Warden Dionaea connector 0.1 for Warden 3.X |
++---------------------------------------------+
+
+Content
+
+  A. Introduction
+  B. Dependencies
+  C. Usage
+  D. Configuration
+
+------------------------------------------------------------------------------
+A. Introduction
+
+   Warden Dionaea connector (executable warden3-dio-sender.py) is a one-shot 
+   script to send events from Dionaea honeypot toward the Warden server.
+
+------------------------------------------------------------------------------
+B. Dependencies
+
+ 1. Platform
+
+    Python 2.7+
+
+ 2. Python packages
+
+    warden_client 3.0+
+
+------------------------------------------------------------------------------
+C. Usage
+
+   warden3-dio-sender.py 
+
+   This script does not run as a daemon, for regularly run use job scheduler cron.
+
+------------------------------------------------------------------------------
+D. Configuration
+   
+   warden_client-dio.cfg
+    warden - path to warden-client config, e.g. 'warden/warden_client.cfg'
+    name - sensor's source id used as a source of events, e.g. 'cz.cesnet.server.dionaea'
+
+    dbfile - path to sqlite database file, e.g. '/opt/dionaea/var/dionaea/logsql.sqlite'
+    binaries_path - path to stored malware, e.g. '/opt/dionaea/var/dionaea/binaries'
+    report_binaries - 'true' if malware attachment have to be included in event, otherwise 'false'
+    con_attempts - number of attempts connection to the database, it may be exclusive locked
+    con_retry_interval - interval between each attempt (in seconds)
+    awin - aggregation window (in minutes), e.g. 5 for events in the last 5 minutes
+    
+   cron
+    SCRIPT_PATH=/opt/warden_client/
+    */5  *   * * *  root cd $SCRIPT_PATH; warden3-dio-sender.py >> dio-sender.log
+
+    Note: Repeat interval must be the same as value of 'awin'.
+
+------------------------------------------------------------------------------
+Copyright (C) 2011-2015 Cesnet z.s.p.o

warden3/contrib/connectors/hp-dio/warden3-dio-sender.py

@@ -4,17 +4,14 @@
 # Copyright (C) 2011-2015 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-from warden_client import Client, Error, read_cfg
+from warden_client import Client, Error, read_cfg, format_timestamp
 import json
 import string
-import sys
 from time import time, gmtime, strftime, sleep
 from math import trunc
 from uuid import uuid4
 from os import path
 import base64
-import pprint
-
 import sqlite3
 
 DEFAULT_ACONFIG = 'warden_client-dio.cfg'
@@ -22,19 +19,14 @@ DEFAULT_WCONFIG = 'warden_client.cfg'
 DEFAULT_BINPATH = '/opt/dionaea/var/dionaea/binaries'
 DEFAULT_DBFILE  = '/opt/dionea/var/dionea/logsql.sqlite'
 DEFAULT_NAME = 'org.example.warden.test'
+DEFAULT_REPORT_BINARIES = 'false'
 DEFAULT_AWIN = 5
 DEFAULT_CON_ATTEMPTS = 3
 DEFAULT_CON_RETRY_INTERVAL = 5
 DEFAULT_ATTACH_NAME = 'att1'
+DEFAULT_HASHTYPE = 'md5'
 
 
-def get_precise_timestamp(epoch=None):
-    t = epoch if epoch else time()
-    us = trunc((t-trunc(t))*1000000)
-    g = gmtime(t)
-    iso = '%04d-%02d-%02dT%02d:%02d:%02d.%0dZ' % (g[0:6]+(us,))
-    return iso
-
 def gen_attach_idea(logger, binaries_path, filename, hashtype, hashdigest, vtpermalink, avref):
     
   refs = []
@@ -73,7 +65,7 @@ def gen_attach_idea(logger, binaries_path, filename, hashtype, hashdigest, vtper
 
   return attach
 
-def gen_event_idea(logger, binaries_path, client_name, detect_time, win_start_time, win_end_time, aggr_win, **data):
+def gen_event_idea(logger, binaries_path, report_binaries, client_name, detect_time, win_start_time, win_end_time, aggr_win, **data):
 
   category = []
   event = {
@@ -90,7 +82,7 @@ def gen_event_idea(logger, binaries_path, client_name, detect_time, win_start_ti
            "Name": client_name,
            "Tags": ["Connection","Honeypot","Recon"],
            "SW": ["Dionaea"],
-           "AggrWin": strftime("%H:%M:%S", gmtime(aggr_win * 60))
+           "AggrWin": strftime("%H:%M:%S", gmtime(aggr_win))
         }
      ]
   }
@@ -118,26 +110,26 @@ def gen_event_idea(logger, binaries_path, client_name, detect_time, win_start_ti
   else:
     category.append('Recon.Scanning')
 
-  # SMB allows save malware
+  # smbd allows save malware
   if data['service'] == 'smbd' and data['download_md5_hash'] is not None:
+    category.append('Malware')
     event['Source'][0]['URL'] = [data['download_url']]
 
-    filename = data['download_url'].split('/')[-1]
-    
-    # Generate "Attach" part of IDEA
-    a = gen_attach_idea(logger, binaries_path, filename, "md5", data['download_md5_hash'], data['virustotal_permalink'], data['scan_result'])
-    
-    # Attach malware section if file with was loaded successfully
-    if a is not None:
-      category.append('Malware')
-      event['Source'][0]['AttachHand'] = [DEFAULT_ATTACH_NAME]
-      event.update(a)
+    if report_binaries == 'true':
+      filename = data['download_url'].split('/')[-1]
+      
+      # Generate "Attach" part of IDEA
+      a = gen_attach_idea(logger, binaries_path, filename, DEFAULT_HASHTYPE, data['download_md5_hash'], data['virustotal_permalink'], data['scan_result'])
+      
+      # Attach malware section if file with malware was loaded successfully
+      if a is not None:
+        event['Source'][0]['AttachHand'] = [DEFAULT_ATTACH_NAME]
+        event.update(a)
 
   event['Source'][0]['Port'] = [data['src_port']]
   event['Target'][0]['Port'] = [data['dst_port']]
   event['Category'] = category
 
-  pprint.pprint(event)
   return event
 
 def main():
@@ -151,6 +143,7 @@ def main():
     adbfile = aconfig.get('dbfile', DEFAULT_DBFILE)
     aconattempts = aconfig.get('con_attempts', DEFAULT_CON_ATTEMPTS)
     aretryinterval = aconfig.get('con_retry_interval', DEFAULT_CON_RETRY_INTERVAL)
+    areportbinaries = aconfig.get('report_binaries', DEFAULT_REPORT_BINARIES)
     
     wconfig['name'] = aname
 
@@ -162,23 +155,14 @@ def main():
 
     events = []
     
-    # query =  "SELECT c.connection_timestamp AS timestamp, c.remote_host AS src_ip, c.remote_port AS src_port, c.connection_transport AS proto, \
-    #           c.local_host AS dst_ip, c.local_port AS dst_port, COUNT(c.connection) as attack_scale, c.connection_protocol AS service, d.download_url, d.download_md5_hash, \
-    #           v.virustotal_permalink, GROUP_CONCAT('urn:' || vt.virustotalscan_scanner || ':' || vt.virustotalscan_result,';') AS scan_result \
-    #           FROM connections AS c LEFT JOIN downloads AS d ON c.connection = d.connection \
-    #           LEFT JOIN virustotals AS v ON d.download_md5_hash = v.virustotal_md5_hash \
-    #           LEFT JOIN virustotalscans vt ON v.virustotal = vt.virustotal \
-    #           WHERE datetime(connection_timestamp,'unixepoch') > datetime('now','-%d minutes') AND c.remote_host != '' \
-    #           GROUP BY c.remote_host, c.local_port ORDER BY c.connection_timestamp ASC;" % (awin)
-
     query =  "SELECT c.connection_timestamp AS timestamp, c.remote_host AS src_ip, c.remote_port AS src_port, c.connection_transport AS proto, \
               c.local_host AS dst_ip, c.local_port AS dst_port, COUNT(c.connection) as attack_scale, c.connection_protocol AS service, d.download_url, d.download_md5_hash, \
               v.virustotal_permalink, GROUP_CONCAT('urn:' || vt.virustotalscan_scanner || ':' || vt.virustotalscan_result,';') AS scan_result \
               FROM connections AS c LEFT JOIN downloads AS d ON c.connection = d.connection \
               LEFT JOIN virustotals AS v ON d.download_md5_hash = v.virustotal_md5_hash \
               LEFT JOIN virustotalscans vt ON v.virustotal = vt.virustotal \
-              WHERE c.remote_host != '' \
-              GROUP BY c.remote_host, c.local_port ORDER BY c.connection_timestamp ASC;"
+              WHERE datetime(connection_timestamp,'unixepoch') > datetime('now','-%d seconds') AND c.remote_host != '' \
+              GROUP BY c.remote_host, c.local_port ORDER BY c.connection_timestamp ASC;" % (awin)
 
     attempts = 0
     while attempts < aconattempts:
@@ -198,21 +182,21 @@ def main():
     if con:
       con.close
 
-    etime = get_precise_timestamp(time())
-    stime = get_precise_timestamp(time() - awin * 60)
+    etime = format_timestamp(time())
+    stime = format_timestamp(time() - awin)
 
     for row in rows:
-      dtime = get_precise_timestamp(row['timestamp'])
-      events.append(gen_event_idea(logger = wclient.logger, binaries_path = abinpath, client_name = aname, detect_time = dtime, win_start_time = stime, win_end_time = etime, aggr_win = awin, **row))
+      dtime = format_timestamp(row['timestamp'])
+      events.append(gen_event_idea(logger = wclient.logger, binaries_path = abinpath, report_binaries = areportbinaries, client_name = aname, detect_time = dtime, win_start_time = stime, win_end_time = etime, aggr_win = awin, **row))
         
-    # print "=== Sending ==="
-    # start = time()
-    # ret = wclient.sendEvents(events)
+    print "=== Sending ==="
+    start = time()
+    ret = wclient.sendEvents(events)
     
-    # if ret:
-    #   wclient.logger.info("%d event(s) successfully delivered." % len(rows))
+    if ret:
+      wclient.logger.info("%d event(s) successfully delivered." % len(rows))
 
-    # print "Time: %f" % (time() - start)
+    print "Time: %f" % (time() - start)
   
   
 

warden3/contrib/connectors/hp-dio/warden_client-dio.cfg

 {
-    "warden": "warden_client.cfg",
-    "name": "cz.cesnet.server.dionaea",
+	"warden": "warden_client.cfg",
+    "name": "cz.cesnet.kryten.dionaea",
 
     "dbfile": "/opt/dionaea/var/dionaea/logsql.sqlite",
     "binaries_path" : "/opt/dionaea/var/dionaea/binaries",
+    "report_binaries" : "true",
     "con_attempts" : 3,
     "con_retry_interval" : 5,
-    "awin": 120
-}
+    "awin": 5
+}
\ No newline at end of file