Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
W
Warden Connectors
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
713
Warden
Warden Connectors
Commits
6743798f
Commit
6743798f
authored
2 years ago
by
Pavel Kácha
Browse files
Options
Downloads
Plain Diff
Merge branch 'backport_nsharp2flowmon_ads'
parents
e4659963
ae0b3bf5
No related branches found
No related tags found
No related merge requests found
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
flowmon-ads/warden3_flowmon_ads_filer.py
+55
-44
55 additions, 44 deletions
flowmon-ads/warden3_flowmon_ads_filer.py
with
55 additions
and
44 deletions
flowmon-ads/warden3_flowmon_ads_filer.py
+
55
−
44
View file @
6743798f
...
...
@@ -4,20 +4,19 @@
# Copyright (C) 2011-2015 Cesnet z.s.p.o
# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
import
os
import
sys
import
getopt
sys
.
path
.
append
(
'
/data/warden/libs
'
)
from
warden_client
import
read_cfg
,
format_time
from
warden_filer
import
SafeDir
import
socket
import
json
import
csv
from
time
import
strptime
,
mktime
import
time
import
re
from
uuid
import
uuid4
sys
.
path
.
append
(
'
/data/warden/libs
'
)
from
warden_client
import
format_time
from
warden_filer
import
SafeDir
# Command line options handling
# Had to use getopt for 2.6 compatibility. Meh. :(
...
...
@@ -55,16 +54,27 @@ def get_opts():
# Conversion routines
def
iso_time
(
t
):
return
strptime
(
t
,
"
%Y-%m-%d %H:%M:%S
"
)
return
time
.
strptime
(
t
,
"
%Y-%m-%d %H:%M:%S
"
)
def
int_list
(
il
):
if
not
il
:
return
[]
return
[
int
(
i
.
strip
())
for
i
in
il
.
split
(
"
,
"
)]
def
str_list
(
sl
):
def
is_ip
(
s
):
try
:
socket
.
inet_pton
(
socket
.
AF_INET
,
s
)
except
Exception
:
try
:
socket
.
inet_pton
(
socket
.
AF_INET6
,
s
)
except
Exception
:
return
False
return
True
def
ip_list
(
sl
):
if
sl
:
return
[
s
.
strip
()
for
s
in
sl
.
split
(
"
,
"
)]
l
=
[
s
.
strip
().
split
(
None
,
1
)[
0
]
for
s
in
sl
.
split
(
"
,
"
)]
return
[
ip
for
ip
in
l
if
is_ip
(
ip
)]
else
:
return
[]
...
...
@@ -88,7 +98,7 @@ def get_proto(s):
return
proto
def
one_proto_list
(
s
):
if
s
:
if
s
and
s
!=
"
not
"
:
return
[
s
]
return
None
...
...
@@ -97,8 +107,8 @@ def proto_list(pl):
ads_fields
=
(
(
'
ID
'
,
int
),
# Unique id within ADS db
(
'
Timestamp
'
,
iso_time
),
# Timestamp of event generation
(
'
ID
'
,
str
),
# Unique id within ADS db
(
'
Detection time
'
,
iso_time
),
# Timestamp of event generation
(
'
FirstFlow
'
,
iso_time
),
# Timestamp of the first Flow on which was based the event detection
(
'
Type
'
,
str
),
# Type of event, in fact a reference to the detection method, which recognized the event
(
'
TypeDesc
'
,
str
),
# Event type description
...
...
@@ -107,9 +117,9 @@ ads_fields = (
(
'
Detail
'
,
str
),
# Detailed information on the event
(
'
Ports
'
,
int_list
),
# List of ports (if identified)
(
'
Protocol
'
,
proto_list
),
# IP protocol (if identified)
(
'
Source
'
,
str
_list
),
# Event originator (IP address)
(
'
Source
'
,
ip
_list
),
# Event originator (IP address)
(
'
CapturedSource
'
,
str
),
# DNS name assigned to the IP address at the time of event detection
(
'
Targets
'
,
str
_list
),
# Event targets (a list of IP addresses)
(
'
Targets
'
,
ip
_list
),
# Event targets (a list of IP addresses)
(
'
NetFlowSource
'
,
str
),
# Flow data source on which the event has been generated
(
'
UserIdentity
'
,
str
)
# User ID from domain controller
)
...
...
@@ -163,7 +173,7 @@ detail_regexps = {
(
INTEGER
+
_SPCM
+
r
"
times
"
,
int
),
(
r
"
connections:
"
+
_SPC
+
INTEGER
,
int
),
(
r
"
Mail count:
"
+
_SPC
+
INTEGER
,
int
),
(
r
"
with response:
"
+
_SPC
+
INTEGER
,
r
"
without response
"
+
_SPC
+
INTEGER
,
sum_int
)
(
r
"
with response:
"
+
_SPC
+
INTEGER
,
r
"
without response
:
"
+
_SPC
+
INTEGER
,
sum_int
)
),
"
byte_count
"
:
(
(
r
"
uploaded:
"
+
_SPC
+
FLOAT
+
_SPC
+
UNITS
,
r
"
downloaded:
"
+
_SPC
+
FLOAT
+
_SPC
+
UNITS
,
sum_int
),
...
...
@@ -207,7 +217,8 @@ unit_translate = {
'
gib
'
:
1024
**
3
}
def
parse_detail
(
e
):
def
parse_detail
(
e
,
log
=
sys
.
stderr
.
write
):
detail
=
e
[
"
Detail
"
]
for
key
,
matchers
in
detail_regexps
.
items
():
results
=
[]
...
...
@@ -233,7 +244,7 @@ def parse_detail(e):
try
:
res
=
shaper
(
reg_res
)
except
Exception
as
e
:
sys
.
stderr
.
write
(
'
Error parsing
"
%s
"
on detail
"
%s
"
: %s
\n
'
%
(
reg_res
,
detail
,
e
))
log
(
'
Error parsing
"
%s
"
on detail
"
%s
"
: %s
\n
'
%
(
reg_res
,
detail
,
e
))
else
:
results
.
append
(
res
)
uniq_results
=
[]
# We cannot use sets for uniq, as result may be unhashable
...
...
@@ -241,24 +252,24 @@ def parse_detail(e):
if
val
and
val
not
in
uniq_results
:
uniq_results
.
append
(
val
)
if
len
(
uniq_results
)
>
1
:
sys
.
stderr
.
write
(
'
Warning: multiple regexp rules matched differently for
"
%s
"
on detail
"
%s
"
\n
'
%
(
key
,
detail
))
log
(
'
Warning: multiple regexp rules matched differently for
"
%s
"
on detail
"
%s
"
\n
'
%
(
key
,
detail
))
if
uniq_results
:
e
[
key
]
=
uniq_results
[
0
]
def
idea_ip_key
(
ip
):
if
not
'
:
'
in
ip
:
if
'
:
'
not
in
ip
:
return
"
IP4
"
else
:
return
"
IP6
"
def
gen_idea_from_ads
(
ads
,
orig_data
,
anonymised_target
,
add_test
):
def
gen_idea_from_ads
(
new_id
,
ads
,
orig_data
,
anonymised_target
,
add_test
):
lts
=
time
.
local
time
()
ts
=
ads
.
get
(
"
Timestamp
"
)
or
lts
ets
=
ads
.
get
(
"
FirstFlow
"
,
0
)
if
ets
>
ts
:
# ADS sometimes reports FirstFlow greater than DetectTime
lts
=
time
.
gm
time
()
ts
=
ads
.
get
(
"
Detection time
"
)
or
lts
ets
=
ads
.
get
(
"
FirstFlow
"
)
if
ets
and
ets
>
ts
:
# ADS sometimes reports FirstFlow greater than DetectTime
ts
=
ets
atype
=
ads
.
get
(
"
Type
"
)
cat
=
ads_types
.
get
(
atype
,
(
"
Other
"
,))[:]
...
...
@@ -276,7 +287,7 @@ def gen_idea_from_ads(ads, orig_data, anonymised_target, add_test):
# Also add some protocols guessed from port and how ADS works according to docs
if
atype
in
(
"
DOS
"
,
"
SIPFLOOD
"
,
"
HTTPDICT
"
):
ads
[
"
Source
"
],
ads
[
"
Targets
"
]
=
ads
[
"
Targets
"
],
ads
[
"
Source
"
]
ads
[
"
CapturedSource
"
],
ads
[
"
target_hostname
"
]
=
None
,
ads
[
"
CapturedSource
"
]
ads
[
"
CapturedSource
"
],
ads
[
"
target_hostname
"
]
=
None
,
ads
.
get
(
"
CapturedSource
"
)
if
atype
==
"
HTTPDICT
"
:
# A guess, sure
if
80
in
ads
[
"
Ports
"
]:
...
...
@@ -304,7 +315,7 @@ def gen_idea_from_ads(ads, orig_data, anonymised_target, add_test):
for
p
in
ads
[
"
Protocol
"
]:
if
p
not
in
proto
:
proto
.
append
(
p
)
ads
[
"
Protocol
"
]
=
proto
ads
[
"
Protocol
"
]
=
proto
or
[
"
tcp
"
]
# Oh well.
# More specific category for BLACKLIST
if
"
botnet
"
in
ads
:
...
...
@@ -315,14 +326,14 @@ def gen_idea_from_ads(ads, orig_data, anonymised_target, add_test):
event
=
{
"
Format
"
:
"
IDEA0
"
,
"
ID
"
:
str
(
uuid4
())
,
"
ID
"
:
new_id
,
"
Category
"
:
cat
,
"
DetectTime
"
:
format_time
(
*
ts
[
0
:
6
]),
"
CreateTime
"
:
format_time
(
*
lts
[
0
:
6
])
}
if
ads
.
get
(
"
ID
"
):
event
[
"
AltNames
"
]
=
[
"
ADS-%i
"
%
ads
[
"
ID
"
]]
event
[
"
AltNames
"
]
=
[
"
ads:%s
"
%
ads
[
"
ID
"
]]
if
ets
:
event
[
"
EventTime
"
]
=
format_time
(
*
ets
[
0
:
6
])
if
ads
.
get
(
"
TypeDesc
"
):
...
...
@@ -339,7 +350,7 @@ def gen_idea_from_ads(ads, orig_data, anonymised_target, add_test):
source
=
{}
for
srcip
in
ads
[
"
Source
"
]:
source
.
setdefault
(
idea_ip_key
(
srcip
),
[]).
append
(
srcip
)
if
ads
[
"
CapturedSource
"
]
:
if
ads
.
get
(
"
CapturedSource
"
)
:
source
[
"
Hostname
"
]
=
[
ads
[
"
CapturedSource
"
]]
if
source
:
source
[
"
Proto
"
]
=
ads
[
"
Protocol
"
]
...
...
@@ -353,7 +364,7 @@ def gen_idea_from_ads(ads, orig_data, anonymised_target, add_test):
if
ads
[
"
Ports
"
]:
target
[
"
Port
"
]
=
ads
[
"
Ports
"
]
if
anonymised_target
!=
"
NONE
"
:
if
anonymised_target
and
anonymised_target
!=
"
NONE
"
:
tgtips
=
[
anonymised_target
]
target
[
"
Type
"
]
=
[
"
Anonymised
"
]
else
:
...
...
@@ -414,7 +425,7 @@ def main():
# Ignore "End of attack" events as they summarise previous ones
# and we would get duplicate counts.
continue
event
=
gen_idea_from_ads
(
tr_row
,
row
,
opts
[
"
target
"
],
opts
[
"
test
"
])
event
=
gen_idea_from_ads
(
str
(
uuid4
()),
tr_row
,
row
,
opts
[
"
target
"
],
opts
[
"
test
"
])
nf
=
out
.
newfile
()
try
:
data
=
json
.
dumps
(
event
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment