Optional incorporating of original line into resulting event is now possible

flowmon-ads/warden3_flowmon_ads_filer.py

@@ -141,7 +141,7 @@ def xlat_ads_proto(s):
     return proto
 
 
-def gen_idea_from_ads(ads, anonymised_target):
+def gen_idea_from_ads(ads, orig_data, anonymised_target):
 
     # Mandatory
     ts = ads["Timestamp"] or time.localtime()
@@ -197,6 +197,13 @@ def gen_idea_from_ads(ads, anonymised_target):
             key = "IP6"
         target.setdefault(key, []).append(tgtip)
 
+    if orig_data:
+        event["Attach"] = [{
+            "Content": "\t".join(orig_data),
+            "Type": ["OrigData"],
+            "ContentType": "text/csv"
+        }]
+
     # Insert subnodes into event
     if source:
         event["Source"] = [source]
@@ -209,13 +216,19 @@ def gen_idea_from_ads(ads, anonymised_target):
 
 def main():
     anonymised_target = None
+    add_orig_data = True
     sdir = SafeDir("/data/warden/var/feeds-out")
 
     ads_fields = [it[0] for it in sorted(ads_fieldnames.items(), key=lambda(it): it[1]["order"])]
-    for row in csv.DictReader(sys.stdin, dialect="excel-tab", fieldnames=ads_fields):
-        for k in row.keys():
-            row[k] = xlat_ads_field(k, row[k])
-        event = gen_idea_from_ads(row, anonymised_target)
+    for row in csv.reader(sys.stdin, dialect="excel-tab"):
+        if not row:
+            continue
+        tr_row = {}
+        for k, val in zip(ads_fields, row):
+            tr_row[k] = xlat_ads_field(k, val)
+        if not add_orig_data:
+            row = None
+        event = gen_idea_from_ads(tr_row, row, anonymised_target)
         nf = sdir.newfile()
         try:
             data = json.dumps(event)