IDEA generation fixes, some adjustments to work with new Dionaea 0.8.0

dionaea/warden3-dio-sender.py

 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2011-2015 Cesnet z.s.p.o
+# Copyright (C) 2011-2018 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
 from warden_client import Client, Error, read_cfg, format_timestamp
@@ -11,45 +11,60 @@ import urllib
 from time import time, gmtime, strftime, sleep
 from math import trunc
 from uuid import uuid4
-from os import path
+from os import path, remove
 import base64
 import sqlite3
 import sys
+import re
+
+aconfig = read_cfg('warden_client_dio.cfg')
+wconfig = read_cfg('warden_client.cfg')
+aclient_name = aconfig['name']
+wconfig['name'] = aclient_name
+aanonymised = aconfig['anonymised']
+aanonymised_net  = aconfig['target_net']
+aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
+atest = aconfig['test_mode']
+
+awin = aconfig['awin'] * 60
+abinpath = aconfig['binaries_path']
+adbfile = aconfig['dbfile']
+aconattempts = aconfig['con_attempts']
+aretryinterval = aconfig['con_retry_interval']
+areportbinaries = aconfig['report_binaries']
+apurgebinaries =  aconfig['purge_binaries']
+
+wconfig['secret'] = aconfig.get('secret', '')
+wclient = Client(**wconfig)
+
+def idea_fill_addresses(event, source_ip, destination_ip, anonymised, anonymised_net):
+   af = "IP4" if not ':' in source_ip else "IP6"
+   event['Source'][0][af] = [source_ip]
+   if anonymised != 'omit':
+           if anonymised == 'yes':
+                   event['Target'][0]['Anonymised'] = True
+                   event['Target'][0][af] = [anonymised_net]
+           else:
+                   event['Target'][0][af] = [destination_ip]
+
+   return event
+
+def gen_attach_idea_smb(logger, report_binaries, purge_binaries, binaries_path, filename, hashtype, hashdigest, vtpermalink, avref):
 
-DEFAULT_ACONFIG = 'warden_client-dio.cfg'
-DEFAULT_WCONFIG = 'warden_client.cfg'
-DEFAULT_BINPATH = '/opt/dionaea/var/dionaea/binaries'
-DEFAULT_DBFILE  = '/opt/dionea/var/dionea/logsql.sqlite'
-DEFAULT_NAME = 'org.example.warden.test'
-DEFAULT_REPORT_BINARIES = 'false'
-DEFAULT_AWIN = 5
-DEFAULT_CON_ATTEMPTS = 3
-DEFAULT_CON_RETRY_INTERVAL = 5
-DEFAULT_ATTACH_NAME = 'att1'
-DEFAULT_HASHTYPE = 'md5'
-DEFAULT_CONTENT_TYPE = 'application/octet-stream'
-DEFAULT_CONTENT_ENCODING = 'base64'
-DEFAULT_ANONYMISED = 'no'
-DEFAULT_TARGET_NET = '0.0.0.0/0'
-DEFAULT_SECRET = ''
-
-
-def gen_attach_idea(logger, report_binaries, binaries_path, filename, hashtype, hashdigest, vtpermalink, avref):
-    
   refs = []
-  attach = { 
-         "Handle": DEFAULT_ATTACH_NAME,
+  attach = {
+         "Handle": 'att1',
          "FileName": [filename],
          "Type": ["Malware"],
          "Hash": ["%s:%s" % (hashtype, hashdigest)],
       }
-  
+
   if vtpermalink is not None:
     refs.append('url:' + vtpermalink)
-  
+
   if avref is not None:
     refs.extend(avref.split(';'))
-  
+
   if refs:
     refs = [urllib.quote(ref, safe=':') for ref in refs]
     refs = list(set(refs))
@@ -60,16 +75,33 @@ def gen_attach_idea(logger, report_binaries, binaries_path, filename, hashtype,
       fpath = path.join(binaries_path, hashdigest)
       with open(fpath, "r") as f:
         fdata = f.read()
-        attach['ContentType'] = DEFAULT_CONTENT_TYPE
-        attach['ContentEncoding'] = DEFAULT_CONTENT_ENCODING
+        attach['ContentType'] = 'application/octet-stream'
+        attach['ContentEncoding'] = 'base64'
         attach['Size'] = len(fdata)
         attach['Content'] = base64.b64encode(fdata)
     except (IOError) as e:
       logger.info("Reading id file \"%s\" with malware failed, information will not be attached." % (fpath))
+    if purge_binaries == 'true':
+      try:
+        remove(filename)
+      except OSError:
+        pass
+
+  return attach
+
+def gen_attach_idea_db(logger, data):
+
+  attach = {}
+  attach["Handle"] = 'att1'
+  attach["Type"] = ["Malware"]
+  attach['ContentType'] = 'application/octet-stream'
+  attach['ContentEncoding'] = 'base64'
+  attach['Size'] = len(data)
+  attach['Content'] = base64.b64encode(data)
 
   return attach
 
-def gen_event_idea(logger, binaries_path, report_binaries, client_name, anonymised, target_net, detect_time, win_start_time, win_end_time, aggr_win, data):
+def gen_event_idea_dio(logger, binaries_path, report_binaries, purge_binaries, client_name, anonymised, target_net, detect_time, win_start_time, win_end_time, aggr_win, data):
 
   category = []
   event = {
@@ -84,105 +116,100 @@ def gen_event_idea(logger, binaries_path, report_binaries, client_name, anonymis
      "Node": [
         {
            "Name": client_name,
-           "Type": ["Connection","Honeypot","Recon"],
+           "Type": ["Connection", "Protocol", "Honeypot"],
            "SW": ["Dionaea"],
            "AggrWin": strftime("%H:%M:%S", gmtime(aggr_win))
         }
      ]
   }
 
-  # Determine IP address family
-  af = "IP4" if not ':' in data['src_ip'] else "IP6"
-  
-  # Extract & save proto and service name
+  # Save TCP/UDP proto
   proto = [data['proto']]
 
-  if data['service'] in ['mysql', 'mssql']:
-    proto.append(data['service'])
-  elif data['service'] in ['httpd', 'smbd']:
-    proto.append(data['service'][:-1])
-
-  # Choose correct category
-  if data['service'] != 'pcap':
-    category.append('Attempt.Exploit')
-  else:
-    category.append('Recon.Scanning')
-
   # smbd allows save malware
   if data['service'] == 'smbd' and data['download_md5_hash'] is not None:
+    category.append('Attempt.Exploit')
     category.append('Malware')
-    event['Source'][0]['URL'] = [data['download_url']]
+    proto.append('smb')
+
+    if data['download_url'] != '':
+        event['Source'][0]['URL'] = [data['download_url']]
     filename = data['download_url'].split('/')[-1]
 
     if filename != '' and data['download_md5_hash'] != '':
-      # Generate "Attach" part of IDEA
-      a = gen_attach_idea(logger, report_binaries, binaries_path, filename, DEFAULT_HASHTYPE, data['download_md5_hash'], data['virustotal_permalink'], data['scan_result'])
-    
-      event['Source'][0]['AttachHand'] = [DEFAULT_ATTACH_NAME]
+      # Generate "SMB Attach" part of IDEA
+      a = gen_attach_idea_smb(logger, report_binaries, binaries_path, filename, "md5", data['download_md5_hash'], data['virustotal_permalink'], data['scan_result'])
+
+      event['Source'][0]['AttachHand'] = ['att1']
       event['Attach'] = [a]
 
+  if data['service'] == 'mysqld':
+    #Clean exported data
+    if data['mysql_query'] is not None:
+          mysql_data = re.sub("select @@version_comment limit 1,?", "", data['mysql_query'])
+          if mysql_data != "":
+            # Generate "MySQL Attach" part of IDEA
+            a = gen_attach_idea_db(logger, mysql_data)
+
+            category.append('Attempt.Exploit')
+            proto.append('mysql')
+            event['Source'][0]['AttachHand'] = ['att1']
+            event['Attach'] = [a]
+
+  if data['service'] == 'mssqld':
+    #Clean exported data
+    if data['mssql_query'] is not None:
+          mssql_data = data['mssql_query']
+          if mssql_data != "":
+            # Generate "MSSQL Attach" part of IDEA
+            a = gen_attach_idea_db(logger, mssql_data)
+
+            category.append('Attempt.Exploit')
+            proto.append('mssql')
+            event['Source'][0]['AttachHand'] = ['att1']
+            event['Attach'] = [a]
 
-  event['Source'][0][af]      = [data['src_ip']]
   event['Source'][0]['Port']  = [data['src_port']]
-
-  if anonymised != 'omit':
-    if anonymised == 'yes':
-      event['Target'][0]['Anonymised'] = True
-      event['Target'][0][af] = [target_net]
-    else:
-      event['Target'][0][af] = [data['dst_ip']]
-
   event['Target'][0]['Port']  = [data['dst_port']]
+  event['Source'][0]['Proto'] = proto
   event['Target'][0]['Proto'] = proto
 
+  idea_fill_addresses(event, data['src_ip'], data['dst_ip'], aanonymised, aanonymised_net)
+
+  # Add default category
+  if not category:
+        category.append('Recon.Scanning')
+
+  # Test if we're testing
+  if atest == "true":
+    category.append('Test')
+
   event['Category'] = category
 
   return event
 
 def main():
-  aconfig = read_cfg(DEFAULT_ACONFIG)
-  wconfig = read_cfg(aconfig.get('warden', DEFAULT_WCONFIG))
-  
-  aname = aconfig.get('name', DEFAULT_NAME)
-  wconfig['name'] = aname   
-
-  asecret = aconfig.get('secret', DEFAULT_SECRET)
-  if asecret:
-    wconfig['secret'] = asecret
-  
-  wclient = Client(**wconfig)
-
-  awin = aconfig.get('awin', DEFAULT_AWIN) * 60
-  abinpath = aconfig.get('binaries_path', DEFAULT_BINPATH)
-  adbfile = aconfig.get('dbfile', DEFAULT_DBFILE)
-  aconattempts = aconfig.get('con_attempts', DEFAULT_CON_ATTEMPTS)
-  aretryinterval = aconfig.get('con_retry_interval', DEFAULT_CON_RETRY_INTERVAL)
-  areportbinaries = aconfig.get('report_binaries', DEFAULT_REPORT_BINARIES)
-  
-  aanonymised = aconfig.get('anonymised', DEFAULT_ANONYMISED)
-  if aanonymised not in ['no', 'yes', 'omit']:
-    wclient.logger.error("Configuration error: anonymised: '%s' - possible typo? use 'no', 'yes' or 'omit'" % aanonymised)
-    sys.exit(2)
-
-  atargetnet  = aconfig.get('target_net', DEFAULT_TARGET_NET)
-  aanonymised = aanonymised if (atargetnet != DEFAULT_TARGET_NET) or (aanonymised == 'omit') else DEFAULT_ANONYMISED
-
-
 
   con = sqlite3.connect(adbfile)
   con.row_factory = sqlite3.Row
+  con.text_factory = str
   crs = con.cursor()
 
   events = []
-  
-  query =  "SELECT c.connection_timestamp AS timestamp, c.remote_host AS src_ip, c.remote_port AS src_port, c.connection_transport AS proto, \
+
+  query = "SELECT c.connection_timestamp AS timestamp, c.remote_host AS src_ip, c.remote_port AS src_port, c.connection_transport AS proto, \
             c.local_host AS dst_ip, c.local_port AS dst_port, COUNT(c.connection) as attack_scale, c.connection_protocol AS service, d.download_url, d.download_md5_hash, \
-            v.virustotal_permalink, GROUP_CONCAT('urn:' || vt.virustotalscan_scanner || ':' || vt.virustotalscan_result,';') AS scan_result \
+            v.virustotal_permalink, GROUP_CONCAT('urn:' || vt.virustotalscan_scanner || ':' || vt.virustotalscan_result,';') AS scan_result, \
+            group_concat(mca.mysql_command_arg_data) as mysql_query, \
+          group_concat(msc.mssql_command_cmd) as mssql_query \
             FROM connections AS c LEFT JOIN downloads AS d ON c.connection = d.connection \
             LEFT JOIN virustotals AS v ON d.download_md5_hash = v.virustotal_md5_hash \
             LEFT JOIN virustotalscans vt ON v.virustotal = vt.virustotal \
-            WHERE datetime(connection_timestamp,'unixepoch') > datetime('now','-%d seconds') AND c.remote_host != '' \
-            GROUP BY c.remote_host, c.local_port ORDER BY c.connection_timestamp ASC;" % (awin)
+            LEFT JOIN mysql_commands mc ON c.connection = mc.connection  \
+            LEFT JOIN mysql_command_args mca ON mc.mysql_command = mca.mysql_command \
+          LEFT JOIN mssql_commands msc ON c.connection = msc.connection \
+          WHERE datetime(connection_timestamp,'unixepoch') > datetime('now','-%d seconds') AND c.remote_host != '' \
+          GROUP BY c.remote_host, c.local_port ORDER BY c.connection_timestamp ASC;" % (awin)
 
   attempts = 0
   while attempts < aconattempts:
@@ -207,16 +234,12 @@ def main():
 
   for row in rows:
     dtime = format_timestamp(row['timestamp'])
-    events.append(gen_event_idea(logger = wclient.logger, binaries_path = abinpath, report_binaries = areportbinaries, client_name = aname, anonymised = aanonymised, target_net = atargetnet, detect_time = dtime, win_start_time = stime, win_end_time = etime, aggr_win = awin, data = row))
-      
-  print "=== Sending ==="
+    events.append(gen_event_idea_dio(logger = wclient.logger, binaries_path = abinpath, report_binaries = areportbinaries, purge_binaries = apurgebinaries, client_name = aclient_name, anonymised = aanonymised, target_net = aanonymised_net, detect_time = dtime, win_start_time = stime, win_end_time = etime, aggr_win = awin, data = row))
+
   start = time()
   ret = wclient.sendEvents(events)
-  
-  if ret:
-    wclient.logger.info("%d event(s) successfully delivered." % len(rows))
-
-  print "Time: %f" % (time() - start)
+  if 'saved' in ret:
+    wclient.logger.info("%d event(s) successfully delivered in %d seconds" % (ret['saved'], (time() - start)))
 
 
 if __name__ == "__main__":